{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "create by xubing on 2020-03-26\n",
    "一种通用的机器学习模型\n",
    "'''\n",
    "\n",
    "import warnings\n",
    "\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split as sp\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "import time\n",
    "\n",
    "\n",
    "def get_pretty_time(seconds):\n",
    "    if seconds == -1:\n",
    "        return 'N/A'\n",
    "\n",
    "    if seconds <= 60:\n",
    "        return '%d秒' % seconds\n",
    "\n",
    "    if seconds <= 3600:\n",
    "        return '%d分钟%d秒' % (seconds // 60, seconds % 60)\n",
    "\n",
    "    return '%d小时%d分' % (seconds // 3600, seconds % 3600 // 60)\n",
    "\n",
    "\n",
    "class Config(object):\n",
    "    def __init__(self):\n",
    "        self.model_config = {\n",
    "            'boosting_type': 'gbdt',  # rf dart goss\n",
    "            'objective': 'binary',  # 任务 可选 {binary（二分类）、multi（多分类）、regression（回归）}\n",
    "            'data_scale': 'middel_small',  # 数据规模\n",
    "            'test_size': 0.2,\n",
    "            'num_class': 3,\n",
    "            'metric': 'f1',  # 可选的参数参见 sklearn.metrics.SCORES.keys()\n",
    "            'folds': 4,\n",
    "            'verbose': 1  # 值越大，显示的log越多 1 5 10 50\n",
    "        }\n",
    "        self.all_params = {\n",
    "            'learning_rate': [0.001, 0.01, 0.1, 0.5, 1],\n",
    "            'n_estimators': [10, 100, 1000],\n",
    "            'subsample': [0.2, 0.4, 0.6, 0.8, 1.0],\n",
    "\n",
    "        }\n",
    "        # self.all_params = {\n",
    "        #     # Step 1 超参数\n",
    "        #     # #max_depth:树深度。 越大，越准确，越慢，越可能过拟合。\n",
    "        #     # #num_leaves: 叶子节点个数 。越大，越准确，越慢、越可能过拟合\n",
    "        #     # #max_bin:工具箱数。（叶子节点+非叶子节点） 越大，越准确，越慢，越可能过拟合。\n",
    "        #     'max_depth': [3, 5, 7, 9, 12, 15, 17, 25, -1],  # 默认 -1\n",
    "        #     'num_leaves': [15, 31, 63, 127, 255],  # 默认 127\n",
    "        #     'max_bin': [63, 127, 255],  # 默认 255\n",
    "        #\n",
    "        #     # Step2 超参数\n",
    "        #     # #feature_fraction:随机选择的特征比例。 越大，越准确，越慢，越可能过拟合。\n",
    "        #     # #bagging_fraction：随机选择的数据比例。 越大，越准确，越慢，越可能过拟合。\n",
    "        #     # #bagging_freq：随机选择数据的频率。越小，越准确，越慢，越可能过拟合。\n",
    "        #     'feature_fraction': [0.6, 0.7, 0.8, 0.9, 1.0],  # 默认 1\n",
    "        #     'bagging_fraction': [0.6, 0.7, 0.8, 0.9, 1.0],  # 默认 1\n",
    "        #     'bagging_freq': range(0, 81, 10),  # 默认0 表示禁用\n",
    "        #\n",
    "        #     # Step 3 超参数\n",
    "        #     # # lambda_l1：L1正则。越小，越准确，越慢，越可能过拟合。\n",
    "        #     # # lambda_l2：L2正则。越小，越准确，越慢，越可能过拟合。\n",
    "        #     'lambda_l1': [1e-5, 1e-3, 1e-1, 0.0, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0],  # 默认 0\n",
    "        #     'lambda_l2': [1e-5, 1e-3, 1e-1, 0.0, 0.1, 0.4, 0.6, 0.7, 0.9, 1.0],  # 默认 0\n",
    "        #\n",
    "        #     # Step4 超参数\n",
    "        #     # # min_split_gain:执行切分的最小增益。\n",
    "        #     # #min_data_in_leaf:落在叶子节点的最小数据数量。越小，越准确，越慢，越可能过拟合。\n",
    "        #     'min_split_gain': [1e-5, 1e-3, 1e-1, 0.0, 0.1, 0.4, 0.6, 0.7, 0.9, 1.0],  # 默认 0\n",
    "        #     'min_data_in_leaf': [10, 20, 50, 100, 200],  # 默认 100\n",
    "        #\n",
    "        #     # Step5 超参数\n",
    "        #     # #learning_rate:学习率（步长）。越小，越准确，越慢，越可能过拟合。\n",
    "        #     # #num_iterations:迭代次数。（alias = num_trees, num_rounds）越大，越准确，越慢，越可能过拟合。\n",
    "        #     'learning_rate': [0.001, 0.003, 0.01, 0.03, 0.05, 0.1, 1],  # 默认 0.1\n",
    "        #     'num_iterations': [100, 500, 1000, 2000]  # 默认 100\n",
    "        # }\n",
    "        #\n",
    "\n",
    "\n",
    "class GeneralMLModel:\n",
    "\n",
    "    def __init__(self, X, y):\n",
    "        config = Config().model_config\n",
    "        self.boosting_type = config['boosting_type']\n",
    "        self.objective = config['objective']\n",
    "        self.data_scale = config['data_scale']  # 数据规模 可选{middle_small(middle and small中小规模)、large(大规模)}\n",
    "        self.test_size = config['test_size']\n",
    "        self.metric = config['metric']\n",
    "        self.num_class = config['num_class']  # 多分类可用\n",
    "        self.folds = config['folds']\n",
    "        self.verbose = config['verbose']\n",
    "\n",
    "        self.all_params = Config().all_params\n",
    "        self.trainX, self.testX, self.trainY, self.testY = sp(X, y, test_size=self.test_size)\n",
    "\n",
    "    def lr_reg(self):\n",
    "        reg = LinearRegression().fit(self.trainX, self.trainY)\n",
    "        return reg.score(self.testX, self.testY)\n",
    "\n",
    "    def lr_clf(self):\n",
    "        lr_start_time = time.time()\n",
    "        clf = LogisticRegression().fit(self.trainX, self.trainY)\n",
    "        lr_clf_end_time = time.time()\n",
    "        lr_clf_cost_time = lr_clf_end_time - lr_start_time\n",
    "        print('lr clf fit cost time:%s' % get_pretty_time(lr_clf_cost_time))\n",
    "        clf_cv = GridSearchCV(estimator=LogisticRegression(),\n",
    "                              param_grid={},\n",
    "                              scoring=self.metric,\n",
    "                              cv=self.folds,\n",
    "                              n_jobs=-1,\n",
    "                              )\n",
    "        clf_cv.fit(self.trainX, self.trainY)\n",
    "        lr_clf_cv_cost_time = time.time() - lr_clf_end_time\n",
    "        print('lr clr_cv fit cost time:%s' % get_pretty_time(lr_clf_cv_cost_time))\n",
    "        return {\n",
    "            'lr clf score:', f1_score(self.testY, clf.predict(self.testX)),\n",
    "            'lr_cv clf score:', f1_score(self.testY, clf_cv.predict(self.testX)),\n",
    "        }\n",
    "\n",
    "    def gbdt_reg(self):\n",
    "        reg = GradientBoostingRegressor().fit(self.trainX, self.trainY)\n",
    "        return reg.score(self.testX, self.testY)\n",
    "\n",
    "    def gbdt_clf(self):\n",
    "        start_time = time.time()\n",
    "        clf = GradientBoostingClassifier().fit(self.trainX, self.trainY)\n",
    "        gbdt_clf_end_time = time.time()\n",
    "        print('gbdt clf fit cost time:%s' % get_pretty_time(gbdt_clf_end_time - start_time))\n",
    "\n",
    "        clf_cv = GridSearchCV(estimator=GradientBoostingClassifier(),\n",
    "                              param_grid={},\n",
    "                              scoring=self.metric,\n",
    "                              cv=self.folds,\n",
    "                              n_jobs=-1,\n",
    "                              )\n",
    "        clf_cv.fit(self.trainX, self.trainY)\n",
    "        gbdt_clf_cv_end_time = time.time()\n",
    "        print('gbdt clr_cv fit cost time:%s' % get_pretty_time(gbdt_clf_cv_end_time - gbdt_clf_end_time))\n",
    "\n",
    "        # 超参数组合数\n",
    "        num_conbins = 1\n",
    "        for val in self.all_params.values():\n",
    "            num_conbins *= len(val)\n",
    "        print('超参数组合数：', num_conbins)\n",
    "\n",
    "        clf_gscv = GridSearchCV(estimator=GradientBoostingClassifier(),\n",
    "                                param_grid=self.all_params,\n",
    "                                scoring=self.metric,\n",
    "                                cv=self.folds,\n",
    "                                n_jobs=-1,\n",
    "                                )\n",
    "        clf_gscv.fit(self.trainX, self.trainY)\n",
    "        gbdt_clf_gscv_end_time = time.time()\n",
    "        print('gbdt clf_gscv fit cost time:%s' % get_pretty_time(gbdt_clf_gscv_end_time - gbdt_clf_cv_end_time))\n",
    "\n",
    "        # return {\n",
    "        #     'gbdt clf score:', f1_score(self.testY, clf.predict(self.testX)),\n",
    "        #     'gbdt_cv clf score:', f1_score(self.testY, clf_cv.predict(self.testX)),\n",
    "        #     'gbdt_gscv clf score:', f1_score(self.testY, clf_gscv.predict(self.testX)),\n",
    "        # }\n",
    "\n",
    "    def xgboost_clf(self):\n",
    "        xgb = XGBClassifier()\n",
    "        start_time = time.time()\n",
    "        xgboost_clf = xgb.fit(self.trainX, self.trainY)\n",
    "        xgboost_clf_end_time = time.time()\n",
    "        print('xgboost clf cost time:', get_pretty_time(xgboost_clf_end_time - start_time))\n",
    "\n",
    "        xgboost_cv_clf = GridSearchCV(\n",
    "            estimator=xgb,\n",
    "            param_grid={},\n",
    "            scoring=self.metric,\n",
    "            cv=self.folds,\n",
    "            n_jobs=-1,\n",
    "        )\n",
    "        xgboost_cv_clf.fit(self.trainX, self.trainY)\n",
    "        xgboost_cv_clf_end_time = time.time()\n",
    "        print('xgboost clf cv cost time:', get_pretty_time(xgboost_cv_clf_end_time - xgboost_clf_end_time))\n",
    "\n",
    "        xgboost_gscv_clf = GridSearchCV(\n",
    "            estimator=xgb,\n",
    "            param_grid=self.all_params,\n",
    "            scoring=self.metric,\n",
    "            cv=self.folds,\n",
    "            n_jobs=-1,\n",
    "        )\n",
    "        xgboost_gscv_clf.fit(self.trainX, self.trainY)\n",
    "        xgboost_gscv_clf_end_time = time.time()\n",
    "        print('xgboost clf gscv cost time:', get_pretty_time(xgboost_gscv_clf_end_time - xgboost_cv_clf_end_time))\n",
    "\n",
    "    def my_base_line(self):\n",
    "        base_line = {\n",
    "            # 'Linear Regressor R2': self.lr_reg(),\n",
    "            # 'GBDT Regressor R2': self.gbdt_reg(),\n",
    "            # 'Logistic Classifier F1': self.lr_clf(),\n",
    "            'GBDT Classifier F1': self.gbdt_clf(),\n",
    "            'XGBoost Classifier F1': self.xgboost_clf(),\n",
    "        }\n",
    "        # for v in base_line.values():\n",
    "        #     for k_, v_ in v.items():\n",
    "        #         print(k_, ':', v_)\n",
    "\n",
    "\n",
    "    def auto_tune_gbdt(self):\n",
    "        # 判断机器学习任务\n",
    "        # 三种常用算法\n",
    "        if self.objective == 'binary':\n",
    "            objective = 'binary'\n",
    "        elif self.objective == 'multiclass':\n",
    "            objective = 'multiclass'\n",
    "            num_class = self.num_class\n",
    "        elif self.objective == 'multiclassova':\n",
    "            objective = 'multiclassova'\n",
    "            num_class = self.num_class\n",
    "        elif self.objective == 'regresssion':\n",
    "            objective = 'regression'\n",
    "            metric = 'l2'\n",
    "        start_time = time.time()\n",
    "        print('=====改善后======')\n",
    "        lgb_train = lgb.Dataset(self.trainX, self.trainY)\n",
    "        lgb_eval = lgb.Dataset(self.testX, self.testY, reference=lgb_train, free_raw_data=False)\n",
    "\n",
    "        fixed_params = {\n",
    "            'boosting_type': self.boosting_type,\n",
    "            'objective': self.objective,\n",
    "            'metric': self.metric,\n",
    "            'save_binary': True,\n",
    "            'random_state': 43,\n",
    "        }\n",
    "\n",
    "        print('======Step1 Start...======')\n",
    "        # Step1 提高精度 max_depth、num_leaves、max_bin\n",
    "        step1_params = {\n",
    "            'max_depth': self.all_params['max_depth'],\n",
    "            'num_leaves': self.all_params['num_leaves'],\n",
    "            'max_bin': self.all_params['max_bin']\n",
    "        }\n",
    "        gs1 = GridSearchCV(estimator=lgb.LGBMClassifier(**fixed_params),\n",
    "                           param_grid=step1_params,\n",
    "                           scoring=self.metric,\n",
    "                           cv=self.folds,\n",
    "                           n_jobs=4,\n",
    "                           verbose=self.verbose\n",
    "                           )\n",
    "\n",
    "        gs1.fit(self.testX, self.testY)\n",
    "        print(\"======Step1 End and Step1's Result======\")\n",
    "        print(\"Best params:\", gs1.best_params_)\n",
    "        print(\"Best {0} score:{1}\".format(self.metric, gs1.best_score_))\n",
    "        step1_end_time = time.time()\n",
    "        step1_cost_time = step1_end_time - start_time\n",
    "        print('Step1 花费时间 %s' % get_pretty_time(step1_cost_time))\n",
    "\n",
    "        # 更新fixed_params\n",
    "        fixed_params['max_depth'] = gs1.best_params_['max_depth']\n",
    "        fixed_params['num_leaves'] = gs1.best_params_['num_leaves']\n",
    "\n",
    "        print('======Step2 Start...======')\n",
    "        # Step2 降低过拟合 feature_fraction bagging_fraction bagging_freq\n",
    "        step2_params = {\n",
    "            'feature_fraction': self.all_params['feature_fraction'],\n",
    "            'bagging_fraction': self.all_params['bagging_fraction'],\n",
    "            'bagging_freq': self.all_params['bagging_freq']\n",
    "        }\n",
    "        gs2 = GridSearchCV(estimator=lgb.LGBMClassifier(**fixed_params),\n",
    "                           param_grid=step2_params,\n",
    "                           scoring=self.metric,\n",
    "                           cv=self.folds,\n",
    "                           n_jobs=-1,\n",
    "                           verbose=self.verbose\n",
    "                           )\n",
    "        gs2.fit(self.trainX, self.trainY)\n",
    "        print('====== Step2 Result======')\n",
    "        print(\"Best params:\", gs2.best_params_)\n",
    "        print(\"Best {0} score:{1}\".format(self.metric, gs2.best_score_))\n",
    "        step2_end_time = time.time()\n",
    "        step2_cost_time = step2_end_time - step1_end_time\n",
    "        print('Step2 花费时间 %s' % get_pretty_time(step2_cost_time))\n",
    "\n",
    "        # 更新fixed_params\n",
    "        fixed_params['feature_fraction'] = gs2.best_params_['feature_fraction']\n",
    "        fixed_params['bagging_fraction'] = gs2.best_params_['bagging_fraction']\n",
    "        fixed_params['bagging_freq'] = gs2.best_params_['bagging_freq']\n",
    "\n",
    "        print('======Step3 Start...======')\n",
    "        # Step3 降低过拟合 lambda_l1 lambda_l2\n",
    "        step3_params = {\n",
    "            'lambda_l1': self.all_params['lambda_l1'],\n",
    "            'lambda_l2': self.all_params['lambda_l2'],\n",
    "        }\n",
    "        gs3 = GridSearchCV(estimator=lgb.LGBMClassifier(**fixed_params),\n",
    "                           param_grid=step3_params,\n",
    "                           scoring=self.metric,\n",
    "                           cv=self.folds,\n",
    "                           n_jobs=-1,\n",
    "                           verbose=self.verbose\n",
    "                           )\n",
    "        gs3.fit(self.trainX, self.trainY)\n",
    "        print('====== Step3 Result======')\n",
    "        print(\"Best params:\", gs3.best_params_)\n",
    "        print(\"Best {0} score:{1}\".format(self.metric, gs3.best_score_))\n",
    "        step3_end_time = time.time()\n",
    "        step3_cost_time = step3_end_time - step2_end_time\n",
    "        print('Step3 花费时间 %s' % get_pretty_time(step3_cost_time))\n",
    "\n",
    "        # 更新fixed_params\n",
    "        fixed_params['lambda_l1'] = gs3.best_params_['lambda_l1']\n",
    "        fixed_params['lambda_l2'] = gs3.best_params_['lambda_l2']\n",
    "\n",
    "        print('======Step4 Start...======')\n",
    "        # Step4 提升精度+降低过拟合 lambda_l1、lambda_l2\n",
    "        step4_params = {\n",
    "            'min_split_gain': self.all_params['min_split_gain'],\n",
    "            'min_data_in_leaf': self.all_params['min_data_in_leaf'],\n",
    "        }\n",
    "        gs4 = GridSearchCV(estimator=lgb.LGBMClassifier(**fixed_params),\n",
    "                           param_grid=step4_params,\n",
    "                           scoring=self.metric,\n",
    "                           cv=self.folds,\n",
    "                           n_jobs=-1,\n",
    "                           verbose=self.verbose\n",
    "                           )\n",
    "        gs4.fit(self.trainX, self.trainY)\n",
    "        print('====== Step4 Result======')\n",
    "        print(\"Best params:\", gs4.best_params_)\n",
    "        print(\"Best {0} score:{1}\".format(self.metric, gs4.best_score_))\n",
    "        step4_end_time = time.time()\n",
    "        step4_cost_time = step4_end_time - step3_end_time\n",
    "        print('Step4 花费时间 %s' % get_pretty_time(step4_cost_time))\n",
    "\n",
    "        # 更新fixed_params\n",
    "        fixed_params['min_split_gain'] = gs4.best_params_['min_split_gain']\n",
    "        fixed_params['min_data_in_leaf'] = gs4.best_params_['min_data_in_leaf']\n",
    "\n",
    "        print('======Step5 Start...======')\n",
    "        # Step5 提升精度 learning_rate、num_rounds\n",
    "        step5_params = {\n",
    "            'learning_rate': self.all_params['learning_rate'],\n",
    "            'num_iterations': self.all_params['num_iterations'],\n",
    "        }\n",
    "        gs5 = GridSearchCV(\n",
    "            estimator=lgb.LGBMClassifier(**fixed_params),\n",
    "            param_grid=step5_params,\n",
    "            scoring=self.metric,\n",
    "            cv=self.folds,\n",
    "            n_jobs=-1,\n",
    "            verbose=self.verbose\n",
    "        )\n",
    "        gs5.fit(self.trainX, self.trainY)\n",
    "        print('====== Step5 Result======')\n",
    "        print(\"Best params:\", gs5.best_params_)\n",
    "        print(\"Best {0} score:{1}\".format(self.metric, gs5.best_score_))\n",
    "        step5_end_time = time.time()\n",
    "        step5_cost_time = step5_end_time - step4_end_time\n",
    "        print('Step5 花费时间 %s' % get_pretty_time(step5_cost_time))\n",
    "\n",
    "        # 更新fixed_params\n",
    "        fixed_params['learning_rate'] = gs5.best_params_['learning_rate']\n",
    "        fixed_params['num_iterations'] = gs5.best_params_['num_iterations']\n",
    "\n",
    "        all_end_time = time.time()\n",
    "        all_cost_time = all_end_time - start_time\n",
    "        print('5 steps 花费时间:%s' % get_pretty_time(all_cost_time))\n",
    "\n",
    "        print('==========最优超参数如下==========')\n",
    "        for k, v in fixed_params.items():\n",
    "            print(k, ':', v)\n",
    "\n",
    "        print('=======使用最优超参数训练模型...========')\n",
    "        model = lgb.LGBMClassifier(**fixed_params)\n",
    "        model.fit(self.trainX, self.trainY)\n",
    "        print('f1 score:', f1_score(self.testY, model.predict(self.testX)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "create by xubing\n",
    "\n",
    "'''\n",
    "\n",
    "# import pandas as pd\n",
    "#\n",
    "# from general_ml_model import GeneralMLModel\n",
    "#\n",
    "# df = pd.read_csv('bank_train.csv')\n",
    "# target = 'y'\n",
    "# df = df.drop([col for col in df.columns if df[col].dtype == 'object'], axis=1)\n",
    "# X = df.drop(target, axis=1)\n",
    "# y = df[target]\n",
    "#\n",
    "# gm = GeneralMLModel(X, y)\n",
    "# gm.auto_tune_gbdt()\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "# from general_ml_model import GeneralMLModel\n",
    "file = '/data/xb/data/ml-data/100w*50.csv'\n",
    "df = pd.read_csv(file)\n",
    "target = 'model'\n",
    "df = df.drop([col for col in df.columns if df[col].dtype == 'object'], axis=1)\n",
    "X = df.drop(target, axis=1)\n",
    "y = df[target]\n",
    "\n",
    "gm = GeneralMLModel(X, y)\n",
    "\n",
    "gm.my_base_line()\n",
    "gm.auto_tune_gbdt()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
